{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Experiment: \n",
    "\n",
    "Replicate how so dense experiments using the new dynamic sparse framework. Compare results with published in the paper\n",
    "\n",
    "#### Motivation.\n",
    "\n",
    "- Ensure our code has no known bugs before proceeding with further experimentation.\n",
    "- Ensure How so Dense experiments are replicable\n",
    "\n",
    "\n",
    "#### Conclusion\n"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "base_exp_config = dict(\n",
    "    device=\"cuda\",\n",
    "    # ----- dataset related ----\n",
    "    dataset_name=\"PreprocessedGSC\",\n",
    "    data_dir=os.path.expanduser(\"~/nta/datasets/gsc\"),\n",
    "    train_batches_per_epoch=5121,\n",
    "    # batch_size_train=(4, 16),\n",
    "    batch_size_train=16,\n",
    "    batch_size_test=20,  # required to fit the GPU\n",
    "    # ----- network related ----\n",
    "    network=\"GSCHeb\",\n",
    "    percent_on_k_winner=[0.095, 0.125, 0.067],\n",
    "    k_inference_factor=1.5,\n",
    "    boost_strength=[1.5, 1.5, 1.5],\n",
    "    boost_strength_factor=[0.9, 0.9, 0.9],\n",
    "    hidden_neurons_conv=[64, 64],\n",
    "    hidden_neurons_fc=1500,\n",
    "    bias=True,\n",
    "    dropout=False,\n",
    "    batch_norm=True,\n",
    "    # ----- model related ----\n",
    "    model=tune.grid_search(\n",
    "        [\"BaseModel\", \"SparseModel\", \"DSNNWeightedMag\", \"DSNNMixedHeb\"]\n",
    "    ),\n",
    "    optim_alg=\"SGD\",\n",
    "    momentum=0,\n",
    "    learning_rate=0.01,\n",
    "    weight_decay=0.01,\n",
    "    lr_scheduler=\"StepLR\",\n",
    "    lr_gamma=0.9,\n",
    "    on_perc=[1, 1, 0.1, 1],\n",
    "    hebbian_prune_perc=None,\n",
    "    hebbian_grow=False,\n",
    "    weight_prune_perc=0.3,\n",
    "    pruning_early_stop=None,  # 2\n",
    "    # additional validation\n",
    "    test_noise=True,\n",
    "    # debugging\n",
    "    debug_weights=True,\n",
    "    debug_sparse=True,\n",
    ")\n",
    "\n",
    "# ray configurations\n",
    "tune_config = dict(\n",
    "    name=__file__.replace(\".py\", \"\") + \"_test2\",\n",
    "    num_samples=8,\n",
    "    local_dir=os.path.expanduser(\"~/nta/results\"),\n",
    "    checkpoint_freq=0,\n",
    "    checkpoint_at_end=False,\n",
    "    stop={\"training_iteration\": 25},\n",
    "    resources_per_trial={\"cpu\": 1, \"gpu\": 0.25},\n",
    "    loggers=DEFAULT_LOGGERS,\n",
    "    verbose=0,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import tabulate\n",
    "import pprint\n",
    "import click\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from ray.tune.commands import *\n",
    "from nupic.research.frameworks.dynamic_sparse.common.browser import *\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import rcParams\n",
    "\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "\n",
    "import seaborn as sns\n",
    "sns.set(style=\"whitegrid\")\n",
    "sns.set_palette(\"colorblind\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and check data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# exps = ['replicate_hsd_test2']\n",
    "# exps = ['replicate_hsd_debug2']\n",
    "exps = ['replicate_hsd_debug5_2x']\n",
    "# exps = ['replicate_hsd_debug5_2x', 'replicate_hsd_debug6_8x']\n",
    "paths = [os.path.expanduser(\"~/nta/results/{}\".format(e)) for e in exps]\n",
    "df = load_many(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Experiment Name</th>\n",
       "      <th>train_acc_max</th>\n",
       "      <th>train_acc_max_epoch</th>\n",
       "      <th>train_acc_min</th>\n",
       "      <th>train_acc_min_epoch</th>\n",
       "      <th>train_acc_median</th>\n",
       "      <th>train_acc_last</th>\n",
       "      <th>val_acc_max</th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th>val_acc_min</th>\n",
       "      <th>...</th>\n",
       "      <th>momentum</th>\n",
       "      <th>network</th>\n",
       "      <th>on_perc</th>\n",
       "      <th>optim_alg</th>\n",
       "      <th>percent_on_k_winner</th>\n",
       "      <th>pruning_early_stop</th>\n",
       "      <th>test_noise</th>\n",
       "      <th>train_batches_per_epoch</th>\n",
       "      <th>weight_decay</th>\n",
       "      <th>weight_prune_perc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_model=BaseModel</td>\n",
       "      <td>0.950298</td>\n",
       "      <td>23</td>\n",
       "      <td>0.636412</td>\n",
       "      <td>0</td>\n",
       "      <td>0.937262</td>\n",
       "      <td>0.950249</td>\n",
       "      <td>0.967121</td>\n",
       "      <td>24</td>\n",
       "      <td>0.893745</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>GSCHeb</td>\n",
       "      <td>0.775</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0.095667</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "      <td>5121</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1_model=SparseModel</td>\n",
       "      <td>0.948198</td>\n",
       "      <td>24</td>\n",
       "      <td>0.655502</td>\n",
       "      <td>0</td>\n",
       "      <td>0.933161</td>\n",
       "      <td>0.948198</td>\n",
       "      <td>0.962711</td>\n",
       "      <td>24</td>\n",
       "      <td>0.878508</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>GSCHeb</td>\n",
       "      <td>0.775</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0.095667</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "      <td>5121</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2_model=DSNNWeightedMag</td>\n",
       "      <td>0.948247</td>\n",
       "      <td>24</td>\n",
       "      <td>0.622107</td>\n",
       "      <td>0</td>\n",
       "      <td>0.932526</td>\n",
       "      <td>0.948247</td>\n",
       "      <td>0.965116</td>\n",
       "      <td>24</td>\n",
       "      <td>0.876103</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>GSCHeb</td>\n",
       "      <td>0.775</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0.095667</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "      <td>5121</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3_model=DSNNMixedHeb</td>\n",
       "      <td>0.950591</td>\n",
       "      <td>24</td>\n",
       "      <td>0.668245</td>\n",
       "      <td>0</td>\n",
       "      <td>0.934821</td>\n",
       "      <td>0.950591</td>\n",
       "      <td>0.961909</td>\n",
       "      <td>22</td>\n",
       "      <td>0.900561</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>GSCHeb</td>\n",
       "      <td>0.775</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0.095667</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "      <td>5121</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4_model=BaseModel</td>\n",
       "      <td>0.951470</td>\n",
       "      <td>24</td>\n",
       "      <td>0.634801</td>\n",
       "      <td>0</td>\n",
       "      <td>0.938629</td>\n",
       "      <td>0.951470</td>\n",
       "      <td>0.963111</td>\n",
       "      <td>23</td>\n",
       "      <td>0.887731</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>GSCHeb</td>\n",
       "      <td>0.775</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0.095667</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "      <td>5121</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 55 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           Experiment Name  train_acc_max  train_acc_max_epoch  train_acc_min  \\\n",
       "0        0_model=BaseModel       0.950298                   23       0.636412   \n",
       "1      1_model=SparseModel       0.948198                   24       0.655502   \n",
       "2  2_model=DSNNWeightedMag       0.948247                   24       0.622107   \n",
       "3     3_model=DSNNMixedHeb       0.950591                   24       0.668245   \n",
       "4        4_model=BaseModel       0.951470                   24       0.634801   \n",
       "\n",
       "   train_acc_min_epoch  train_acc_median  train_acc_last  val_acc_max  \\\n",
       "0                    0          0.937262        0.950249     0.967121   \n",
       "1                    0          0.933161        0.948198     0.962711   \n",
       "2                    0          0.932526        0.948247     0.965116   \n",
       "3                    0          0.934821        0.950591     0.961909   \n",
       "4                    0          0.938629        0.951470     0.963111   \n",
       "\n",
       "   val_acc_max_epoch  val_acc_min  ...  momentum  network  on_perc  optim_alg  \\\n",
       "0                 24     0.893745  ...         0   GSCHeb    0.775        SGD   \n",
       "1                 24     0.878508  ...         0   GSCHeb    0.775        SGD   \n",
       "2                 24     0.876103  ...         0   GSCHeb    0.775        SGD   \n",
       "3                 22     0.900561  ...         0   GSCHeb    0.775        SGD   \n",
       "4                 23     0.887731  ...         0   GSCHeb    0.775        SGD   \n",
       "\n",
       "   percent_on_k_winner  pruning_early_stop  test_noise  \\\n",
       "0             0.095667                None        True   \n",
       "1             0.095667                None        True   \n",
       "2             0.095667                None        True   \n",
       "3             0.095667                None        True   \n",
       "4             0.095667                None        True   \n",
       "\n",
       "   train_batches_per_epoch  weight_decay weight_prune_perc  \n",
       "0                     5121          0.01               0.3  \n",
       "1                     5121          0.01               0.3  \n",
       "2                     5121          0.01               0.3  \n",
       "3                     5121          0.01               0.3  \n",
       "4                     5121          0.01               0.3  \n",
       "\n",
       "[5 rows x 55 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace hebbian prine\n",
    "df['hebbian_prune_perc'] = df['hebbian_prune_perc'].replace(np.nan, 0.0, regex=True)\n",
    "df['weight_prune_perc'] = df['weight_prune_perc'].replace(np.nan, 0.0, regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Experiment Name', 'train_acc_max', 'train_acc_max_epoch',\n",
       "       'train_acc_min', 'train_acc_min_epoch', 'train_acc_median',\n",
       "       'train_acc_last', 'val_acc_max', 'val_acc_max_epoch', 'val_acc_min',\n",
       "       'val_acc_min_epoch', 'val_acc_median', 'val_acc_last', 'noise_acc_max',\n",
       "       'noise_acc_max_epoch', 'noise_acc_min', 'noise_acc_min_epoch',\n",
       "       'noise_acc_median', 'noise_acc_last', 'val_acc_all', 'epochs',\n",
       "       'experiment_file_name', 'trial_time', 'mean_epoch_time', 'batch_norm',\n",
       "       'batch_size_test', 'batch_size_train', 'bias', 'boost_strength',\n",
       "       'boost_strength_factor', 'data_dir', 'dataset_name', 'debug_sparse',\n",
       "       'debug_weights', 'device', 'dropout', 'hebbian_grow',\n",
       "       'hebbian_prune_perc', 'hidden_neurons_conv', 'hidden_neurons_fc',\n",
       "       'k_inference_factor', 'learning_rate', 'lr_gamma', 'lr_scheduler',\n",
       "       'model', 'momentum', 'network', 'on_perc', 'optim_alg',\n",
       "       'percent_on_k_winner', 'pruning_early_stop', 'test_noise',\n",
       "       'train_batches_per_epoch', 'weight_decay', 'weight_prune_perc'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(120, 55)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Experiment Name                                          1_model=SparseModel\n",
       "train_acc_max                                                       0.948198\n",
       "train_acc_max_epoch                                                       24\n",
       "train_acc_min                                                       0.655502\n",
       "train_acc_min_epoch                                                        0\n",
       "train_acc_median                                                    0.933161\n",
       "train_acc_last                                                      0.948198\n",
       "val_acc_max                                                         0.962711\n",
       "val_acc_max_epoch                                                         24\n",
       "val_acc_min                                                         0.878508\n",
       "val_acc_min_epoch                                                          0\n",
       "val_acc_median                                                      0.952285\n",
       "val_acc_last                                                        0.962711\n",
       "noise_acc_max                                                       0.970219\n",
       "noise_acc_max_epoch                                                       21\n",
       "noise_acc_min                                                     0.00548589\n",
       "noise_acc_min_epoch                                                        8\n",
       "noise_acc_median                                                    0.172414\n",
       "noise_acc_last                                                      0.119122\n",
       "val_acc_all                0     0.878508\n",
       "1     0.906576\n",
       "2     0.916199\n",
       "3...\n",
       "epochs                                                                    25\n",
       "experiment_file_name       /Users/lsouza/nta/results/replicate_hsd_debug5...\n",
       "trial_time                                                           5.59045\n",
       "mean_epoch_time                                                     0.223618\n",
       "batch_norm                                                              True\n",
       "batch_size_test                                                           20\n",
       "batch_size_train                                                          16\n",
       "bias                                                                    True\n",
       "boost_strength                                                           1.5\n",
       "boost_strength_factor                                                    0.9\n",
       "data_dir                                       /home/ubuntu/nta/datasets/gsc\n",
       "dataset_name                                                 PreprocessedGSC\n",
       "debug_sparse                                                            True\n",
       "debug_weights                                                           True\n",
       "device                                                                  cuda\n",
       "dropout                                                                False\n",
       "hebbian_grow                                                           False\n",
       "hebbian_prune_perc                                                         0\n",
       "hidden_neurons_conv                                                       64\n",
       "hidden_neurons_fc                                                       1500\n",
       "k_inference_factor                                                       1.5\n",
       "learning_rate                                                           0.01\n",
       "lr_gamma                                                                 0.9\n",
       "lr_scheduler                                                          StepLR\n",
       "model                                                            SparseModel\n",
       "momentum                                                                   0\n",
       "network                                                               GSCHeb\n",
       "on_perc                                                                0.775\n",
       "optim_alg                                                                SGD\n",
       "percent_on_k_winner                                                0.0956667\n",
       "pruning_early_stop                                                      None\n",
       "test_noise                                                              True\n",
       "train_batches_per_epoch                                                 5121\n",
       "weight_decay                                                            0.01\n",
       "weight_prune_perc                                                        0.3\n",
       "Name: 1, dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "model\n",
       "BaseModel          30\n",
       "DSNNMixedHeb       30\n",
       "DSNNWeightedMag    30\n",
       "SparseModel        30\n",
       "Name: model, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('model')['model'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ## Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Experiment Details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_epochs = 25\n",
    "# Did any  trials failed?\n",
    "df[df[\"epochs\"]<num_epochs][\"epochs\"].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(120, 55)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Removing failed or incomplete trials\n",
    "df_origin = df.copy()\n",
    "df = df_origin[df_origin[\"epochs\"]>=num_epochs]\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Series([], Name: epochs, dtype: int64)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# which ones failed?\n",
    "# failed, or still ongoing?\n",
    "df_origin['failed'] = df_origin[\"epochs\"]<num_epochs\n",
    "df_origin[df_origin['failed']]['epochs']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper functions\n",
    "def mean_and_std(s):\n",
    "    return \"{:.2f} ± {:.2f}\".format(s.mean()*100, s.std()*100)\n",
    "\n",
    "def round_mean(s):\n",
    "    return \"{:.0f}\".format(round(s.mean()))\n",
    "\n",
    "stats = ['min', 'max', 'mean', 'std']\n",
    "\n",
    "def agg(columns, filter=None, round=3):\n",
    "    if filter is None:\n",
    "        return (df.groupby(columns)\n",
    "             .agg({'val_acc_max_epoch': round_mean,\n",
    "                   'val_acc_max': stats,\n",
    "                   'val_acc_last': stats,\n",
    "                   'model': ['count']})).round(round)\n",
    "    else:\n",
    "        return (df[filter].groupby(columns)\n",
    "             .agg({'val_acc_max_epoch': round_mean,\n",
    "                   'val_acc_max': stats,                \n",
    "                   'val_acc_last': stats,\n",
    "                   'model': ['count']})).round(round)\n",
    "    \n",
    "    \n",
    "def agg_paper(columns, filter=None, round=3):\n",
    "    if filter is None:\n",
    "        return (df.groupby(columns)\n",
    "             .agg({'val_acc_max': mean_and_std,\n",
    "                   'val_acc_last': mean_and_std,\n",
    "                   'train_acc_last': mean_and_std,\n",
    "                   'model': ['count']})).round(round)\n",
    "    else:\n",
    "        return (df[filter].groupby(columns)\n",
    "             .agg({'val_acc_max': mean_and_std,\n",
    "                   'val_acc_last': mean_and_std,\n",
    "                   'train_acc_last': mean_and_std,\n",
    "                   'model': ['count']})).round(round)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_last</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BaseModel</th>\n",
       "      <td>22</td>\n",
       "      <td>0.962</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.965</td>\n",
       "      <td>0.002</td>\n",
       "      <td>0.958</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.962</td>\n",
       "      <td>0.002</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>20</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.966</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.002</td>\n",
       "      <td>0.953</td>\n",
       "      <td>0.966</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.003</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>20</td>\n",
       "      <td>0.961</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.002</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.965</td>\n",
       "      <td>0.961</td>\n",
       "      <td>0.003</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SparseModel</th>\n",
       "      <td>21</td>\n",
       "      <td>0.958</td>\n",
       "      <td>0.966</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.002</td>\n",
       "      <td>0.956</td>\n",
       "      <td>0.966</td>\n",
       "      <td>0.961</td>\n",
       "      <td>0.003</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                val_acc_max_epoch val_acc_max                       \\\n",
       "                       round_mean         min    max   mean    std   \n",
       "model                                                                \n",
       "BaseModel                      22       0.962  0.970  0.965  0.002   \n",
       "DSNNMixedHeb                   20       0.960  0.966  0.963  0.002   \n",
       "DSNNWeightedMag                20       0.961  0.968  0.964  0.002   \n",
       "SparseModel                    21       0.958  0.966  0.963  0.002   \n",
       "\n",
       "                val_acc_last                      model  \n",
       "                         min    max   mean    std count  \n",
       "model                                                    \n",
       "BaseModel              0.958  0.967  0.962  0.002    30  \n",
       "DSNNMixedHeb           0.953  0.966  0.960  0.003    30  \n",
       "DSNNWeightedMag        0.954  0.965  0.961  0.003    30  \n",
       "SparseModel            0.956  0.966  0.961  0.003    30  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max</th>\n",
       "      <th>val_acc_last</th>\n",
       "      <th>train_acc_last</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean_and_std</th>\n",
       "      <th>mean_and_std</th>\n",
       "      <th>mean_and_std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BaseModel</th>\n",
       "      <td>96.52 ± 0.18</td>\n",
       "      <td>96.21 ± 0.23</td>\n",
       "      <td>95.14 ± 0.12</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>96.32 ± 0.16</td>\n",
       "      <td>96.04 ± 0.28</td>\n",
       "      <td>94.91 ± 0.14</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>96.37 ± 0.19</td>\n",
       "      <td>96.10 ± 0.31</td>\n",
       "      <td>94.89 ± 0.14</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SparseModel</th>\n",
       "      <td>96.27 ± 0.17</td>\n",
       "      <td>96.07 ± 0.26</td>\n",
       "      <td>94.88 ± 0.16</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  val_acc_max  val_acc_last train_acc_last model\n",
       "                 mean_and_std  mean_and_std   mean_and_std count\n",
       "model                                                           \n",
       "BaseModel        96.52 ± 0.18  96.21 ± 0.23   95.14 ± 0.12    30\n",
       "DSNNMixedHeb     96.32 ± 0.16  96.04 ± 0.28   94.91 ± 0.14    30\n",
       "DSNNWeightedMag  96.37 ± 0.19  96.10 ± 0.31   94.89 ± 0.14    30\n",
       "SparseModel      96.27 ± 0.17  96.07 ± 0.26   94.88 ± 0.16    30"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg_paper(['model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
